package edu.buaa.act

import java.io._
import java.net.URL
import java.util
import java.util.Calendar
import java.nio.channels.Channels
import java.nio.file.Files

import org.apache.commons.compress.archivers.ArchiveEntry
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream
import org.apache.commons.compress.utils.IOUtils

import scala.collection.mutable
//import org.apache.commons.lang3.SystemUtils

import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
import scala.io.Source


object DataReader {
    private val FILE_PATH_SEPARATOR = "\\"
    private val urlHeader = "http://chengjinglearn.qiniudn.com/TGraphDemo/"
    private val fileList = Array[String](
        "20101104.tar.gz", "20101105.tar.gz", "20101106.tar.gz",
        "20101107.tar.gz", "20101108.tar.gz")

    val entityIdMap = new mutable.HashMap[String, Int]()

    private def importFileData(dataFileList: Array[File]): Array[DataEntry] = {
        dataFileList.sorted((o1: File, o2: File) => o1.getName.compareTo(o2.getName))
        entityIdMap.clear()
        val result = new ArrayBuffer[DataEntry]
        for(file <- dataFileList){
            val startTime = timeStr2int(file.getName.substring(9, 21)) - 1288800000
            fileInput(result, file, startTime)
        }
        result.toArray
    }

    private def fileInput(result:ArrayBuffer[DataEntry], file: File, startTime: Int): Unit = {
        val lines:Iterator[String] = Source.fromFile(file).getLines()
        lines.drop(1) // remove header
        for(line <- lines){
            if(line.length>0) result += readDataLine(line, startTime)
        }
    }

    private def readDataLine(line: String, startTime: Int): DataEntry = {
        val fields = line.split(",")
        val gridId = fields(1)
        val chainId = fields(2)
        val entityId = getEntityId(gridId, chainId)
        val travelTime = fields(6).toInt
        val fullStatus = fields(7).toByte
        val vehicleCount = fields(8).toByte
        val segmentCount = fields(9).toInt
        new DataEntry(entityId, startTime, travelTime, fullStatus, vehicleCount, segmentCount)
    }

    private def getEntityId(gridId: String, chainId: String):Int = {
        val key = gridId + ":" + chainId
        if (entityIdMap.contains(key)) entityIdMap(key)
        else {
            val value = entityIdMap.size
            entityIdMap.put(key, value)
            value
        }
    }

    /**
      * the same function as TrafficDataImporter.java->timeStr2int()
      */
    private def timeStr2int(tStr: String):Int = {
        val yearStr = tStr.substring(0, 4)
        val monthStr = tStr.substring(4, 6)
        val dayStr = tStr.substring(6, 8)
        val hourStr = tStr.substring(8, 10)
        val minuteStr = tStr.substring(10, 12)
        val year = yearStr.toInt
        val month = monthStr.toInt - 1
        //month count from 0 to 11, no 12
        val day = dayStr.toInt
        val hour = hourStr.toInt
        val minute = minuteStr.toInt
        val ca = Calendar.getInstance
        ca.set(year, month, day, hour, minute, 0) //seconds set to 0

        val timestamp = ca.getTimeInMillis
        if (timestamp / 1000 < Integer.MAX_VALUE) (timestamp / 1000).toInt
        else throw new RuntimeException("timestamp larger than Integer.MAX_VALUE, this should not happen")
    }

    @throws[IOException]
    def download(count: Int):Unit = {
        val dataDir = new File("data")
        if(!dataDir.exists()){
            Files.createDirectory(dataDir.toPath)
        }
        download(urlHeader+fileList(count), dataDir)
    }

    @throws[IOException]
    private def download(url: String, dir: File): Unit = {
        val out = new File(dir, url.substring(url.length - 8))
        if (out.exists && out.isFile) return
        val website = new URL(url)
        val rbc = Channels.newChannel(website.openStream)
        val fos = new FileOutputStream(out.getAbsolutePath)
        fos.getChannel.transferFrom(rbc, 0, Long.MaxValue)
    }

    /**
      * Tar文件解压方法
      *
      * @param input
      * 要解压的压缩文件名称（绝对路径名称）
      * @param targetDir
      * 解压后文件放置的路径名（绝对路径名称）
      * @return 解压出的文件列表
      */
    @throws[IOException]
    private def decompressTarGZip(input: File, targetDir: File):util.ArrayList[File] = {
        if (!targetDir.isDirectory && !targetDir.mkdirs) throw new IOException("failed to create directory " + targetDir)
        val result = new util.ArrayList[File]
        val i = new TarArchiveInputStream(new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(input))))
        var entry:ArchiveEntry = i.getNextEntry
        while( entry!=null){
            if (i.canReadEntryData(entry)) {
                val name = targetDir.getAbsolutePath + FILE_PATH_SEPARATOR + entry.getName
                val f = new File(name)
                if (entry.isDirectory) if (!f.isDirectory && !f.mkdirs) throw new IOException("failed to create directory " + f)
                else {
                    result.add(f)
                    val parent = f.getParentFile
                    if (!parent.isDirectory && !parent.mkdirs) throw new IOException("failed to create directory " + parent)
                    val o = Files.newOutputStream(f.toPath)
                    IOUtils.copy(i, o)
                    o.close()
                }
            }else{
                println("can not read entry")
            }
            entry = i.getNextEntry
        }
        result
    }

    def getFiles(dir: File, filterFun:File=>Boolean): Array[File] = {
        val d = dir.listFiles.filter(_.isDirectory)
        val f = dir.listFiles.filter(filterFun)
        f ++ d.flatMap(getFiles(_, filterFun))
    }

    def getData(count:Int):Array[DataEntry]={
        val dataDir = new File("data")
        var zipFileCount:Int = getFiles(dataDir, f => f.getName.endsWith("tar.gz")).length
        while(true){
            val files = getFiles(dataDir, f=>f.getName.endsWith("csv"))
            if(files.length < count){
                download(zipFileCount)
                zipFileCount += 1
            }else{
                return importFileData(files)
            }
        }
        throw new RuntimeException("SNH")
    }

    def dataEntry2IndexEntry(data:Array[DataEntry]): Map[String, Map[Int, Array[IndexEntry]]] ={
        data.sortBy(i=>i.start)
        data.groupBy()
        val travelTime = new mutable.HashMap[Long, ArrayBuffer[TimeValuePair]]
        val jamStatus = new mutable.HashMap[Long, ArrayBuffer[TimeValuePair]]
        val segCount = new mutable.HashMap[Long, ArrayBuffer[TimeValuePair]]
        val carCount = new mutable.HashMap[Long, ArrayBuffer[TimeValuePair]]
        for( d <- data){
            if(!(travelTime contains d.entityId)){
                travelTime.put(d.entityId, new ArrayBuffer[TimeValuePair]())
                jamStatus.put(d.entityId, new ArrayBuffer[TimeValuePair])
                segCount.put(d.entityId, new ArrayBuffer[TimeValuePair])
                carCount.put(d.entityId, new ArrayBuffer[TimeValuePair])
                travelTime(d.entityId) += new TimeValuePair(d.start, d.travelTime)
                jamStatus(d.entityId) += new TimeValuePair(d.start, d.jamStatus)
                segCount(d.entityId) += new TimeValuePair(d.start, d.segCount)
                carCount(d.entityId) += new TimeValuePair(d.start, d.carCount)
            }else{
                val t = travelTime(d.entityId)
                val j = jamStatus(d.entityId)
                val s = segCount(d.entityId)
                val c = carCount(d.entityId)
                if(t.last.value!=d.travelTime) t += new TimeValuePair(d.start, d.travelTime)
                if(j.last.value!=d.jamStatus) j += new TimeValuePair(d.start, d.jamStatus)
                if(s.last.value!=d.segCount) s += new TimeValuePair(d.start, d.segCount)
                if(c.last.value!=d(carCount)) c += new TimeValuePair(d.start, d.carCount)
            }
        }
        val t = new ArrayBuffer[IndexEntry]
        val j = new ArrayBuffer[IndexEntry]
        val s = new ArrayBuffer[IndexEntry]
        val c = new ArrayBuffer[IndexEntry]
        for
    }


}
